package com.zhangyx;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.zhangyx.crawler.comm.DBUtil;
import com.zhangyx.crawler.comm.EncryptUtil;
import com.zhangyx.crawler.core.Crawler;
import com.zhangyx.crawler.core.HttpClientUtil;
import com.zhangyx.crawler.core.Page;
import com.zhangyx.crawler.core.SeedManager;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.List;

public class LgCrawler extends Crawler {
    private static Logger log = Logger.getLogger(LgCrawler.class);
    private static int com_id = 1;
    @Override
    public int intervalTime() {
        return 500;
    }

    @Override
    public Page download() {
        int comNum = getComNum();
        if (comNum > 300000) {
            this.interrupt();
        }
        SeedManager.addRepeat("" + comNum);
        Page page = new Page();
        page.putInfo("com_num", comNum);
        String uuid = EncryptUtil.getUUID();
        HttpClientUtil httpClientUtil = HttpClientUtil.GET("https://www.lagou.com/gongsi/" + comNum + ".html")
                .addCookie("user_trace_token", "20171026221342-" + uuid, "www.lagou.com", "/")
                .addCookie("LGUID", "20171026221342-" + uuid, "www.lagou.com", "/")
                .addCookie("SEARCH_ID", "fc265d15eaf6459994012d0a7d914c5b", "www.lagou.com", "/")
                .addCookie("LGSID", "20171026221342-" + uuid, "www.lagou.com", "/")
                .addCookie("LGRID", "20171026221342-" + uuid, "www.lagou.com", "/")
                .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36")
                .fakeIP()
                .start();
        page.setHtml(httpClientUtil.getHtml());
        return page;
    }

    @Override
    public void deal(Page page) {
        try {
            if (page.getHtml() == null) {
                return;
            }
            Document doc = Jsoup.parse(page.getHtml());
//            log.info(page.getHtml());
            Elements elements = doc.select("div.company_data > ul > li:nth-child(1) > strong");
            if (elements.size() > 0) {
                String jobNumStr = elements.get(0).text();
                if (jobNumStr.equals("暂无")) {
                    return;
                }
                jobNumStr = jobNumStr.replaceAll("个", "");
                int totalNum = Integer.parseInt(jobNumStr);
                getJobList(totalNum, (Integer) page.getInfo("com_num"));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void getJobList(int totalNum, int comNum) {
        List<JobEntity> jobEntities = Lists.newArrayList();
        String uuid = EncryptUtil.getUUID();
        int page = 1;
        while ((totalNum - 10 * (page - 1)) > 0) {
            try {
                HttpClientUtil httpClientUtil = HttpClientUtil.POST("https://www.lagou.com/gongsi/searchPosition.json?companyId=" + comNum + "&positionFirstType=%E5%85%A8%E9%83%A8&schoolJob=false&pageNo=" + page + "&pageSize=10")
                        .addCookie("user_trace_token", "20171026221342-" + uuid, "www.lagou.com", "/")
                        .addCookie("LGUID", "20171026221342-" + uuid, "www.lagou.com", "/")
                        .addCookie("SEARCH_ID", "fc265d15eaf6459994012d0a7d914c5b", "www.lagou.com", "/")
                        .addCookie("LGSID", "20171026221342-" + uuid, "www.lagou.com", "/")
                        .addCookie("LGRID", "20171026221342-" + uuid, "www.lagou.com", "/")
                        .addHeader("Referer", "https://www.lagou.com/gongsi/j" + comNum + ".html")
                        .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36")
                        .fakeIP()
                        .start();

                JSONObject jsonObject = JSON.parseObject(httpClientUtil.getHtml()).getJSONObject("content").getJSONObject("data").getJSONObject("page");
                List<JobEntity> jobs = jsonObject.getJSONArray("result").toJavaList(JobEntity.class);
                jobEntities.addAll(jobs);
            } catch (Exception e) {
                e.printStackTrace();
            }
            page++;
        }
        for (JobEntity jobEntity : jobEntities) {
            save(jobEntity);
        }

    }

    private static synchronized int getComNum() {
        do {
            com_id++;
        } while (SeedManager.isRepeat("job" + com_id));
        return com_id;
    }

    private static void save(JobEntity jobEntity) {
        if (jobEntity.getCreateTime().contains(":")) {
            jobEntity.setCreateTime("2017-10-26");
        }
        String sql = "insert into lg_job(companyId,positionId,jobNature,financeStage,companyName" +
                ",companyFullName,companySize,industryField,positionName,city,createTime" +
                ",salary,workYear,education,positionAdvantage) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
        DBUtil.executePreparedSQL(sql, jobEntity.getCompanyId(), jobEntity.getPositionId(), jobEntity.getJobNature(), jobEntity.getFinanceStage()
                , jobEntity.getCompanyName(), jobEntity.getCompanyFullName(), jobEntity.getCompanySize()
                , jobEntity.getIndustryField(), jobEntity.getPositionName(), jobEntity.getCity(), jobEntity.getCreateTime()
                , jobEntity.getSalary(), jobEntity.getWorkYear(), jobEntity.getEducation(), jobEntity.getPositionAdvantage());
    }

    public static void main(String[] args) {

    }
}
